# -*- coding: utf-8 -*-
from urllib import parse
import scrapy
import re
import json

from scrapy.loader import ItemLoader
from cnblogs_reptile.items import ArticleItemLoader
from scrapy import Selector
from scrapy import Request
import requests

from cnblogs_reptile.items import  JobBoleArticleItem
from cnblogs_reptile.utils import common

class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['news.cnblogs.com']
    start_urls = ['https://news.cnblogs.com/']

    def parse(self, response):

        # 1.获取新闻列表页中的url并交给scrapy进行下载后调用相应的解析方法
        # 2.获取到下一页的url并交给scrapy进行下载,下载完成后交给parse继续跟进
        # sel = Selector(text=response.text)
        #urls = response.xpath('//div[@id="news_list"]//h2[@class="news_entry"]/a/@href').extract()
        post_nodes = response.xpath('//div[@id="news_list"]//div[@class="news_block"]')
        #post_nodes = response.css('#news_list .news_block')
        for post_node in post_nodes:
            # post_node.extract()
            image_url = post_node.xpath('.//div[@class="entry_summary"]/a/img/@src').extract_first("")
            # image_url = post_node.css('.entry_summary a img::attr(src)').extract_first("")
            image = ""
            if image_url:
                image = parse.urljoin(response.url, image_url)
            post_url = post_node.xpath('./div[@class="content"]//h2[@class="news_entry"]/a/@href').extract_first("")
            # 通过parse.urljoin为没有添加域名的post_url添加域名，如果有域名则不会处理
            yield Request(url=parse.urljoin(response.url, post_url), meta={"front_image_url":image}, callback=self.parse_detail)
        #/html/body/div[2]/div[2]/div[4]/div[1]/div[2]/h2/a/@href
        #//*[@id="entry_655299"]/div[2]/h2/a
        #//div[@id="news_list"]/div[1]/div[2]/h2/a

        # 提取下一页并交给scrapy进行下载
        # last() 最后一个元素
        next_url = response.xpath('//div[@class="pager"]/a[last()]/text()').extract_first("")
        # 简化if用xpath判断
        # next_url = response.xpath('//a[contains(text(), "Next >")]/@href').extract_first("")
        if next_url == "Next >":
            next_url = response.xpath('//div[@class="pager"]/a[last()]/@href').extract_first("")
            yield Request(url=parse.urljoin(response.url, next_url),
                          callback=self.parse)

        pass
    def parse_detail(self, response):

        # 获取url中的id
        match_re = re.match(".*?(\d+)",response.url)
        if match_re:
            post_id = match_re.group(1)
            #article_item = JobBoleArticleItem()
            #title = response.xpath('//div[@id="news_title"]/a/text()').extract_first("")
            #create_data = response.xpath('//div[@id="news_info"]/*[@class="time"]/text()').extract_first("")
            #match_re = re.match(".*?(\d+.*)", create_data)
            #if match_re:
            #    create_data = match_re.group(1)
            #content = response.xpath('//div[@id="news_content"]').extract()[0]
            #tag_list = response.xpath('//*[@class="news_tags"]/a/text()').extract()
            #tags = ",".join(tag_list)

            # html = requests.get(parse.urljoin(response.url, "/NewsAjax/GetAjaxNewsInfo?contentId={}".format(post_id)))

            # j_data = json.loads(html.text)

            #article_item["url"] = response.url
            #article_item["title"] = title
            #article_item["create_data"] = create_data
            #article_item["content"] = content
            #article_item["tags"] = tags
            # 图片下载的对象应该存入list
            #if response.meta.get("front_image_url", ""):
            #    article_item["front_image_url"] = [response.meta.get("front_image_url", "")]
            #else:
            #    article_item["front_image_url"] = []

            item_loader = ArticleItemLoader(item=JobBoleArticleItem(), response=response)
            item_loader.add_xpath("title", '//div[@id="news_title"]/a/text()')
            item_loader.add_xpath("content", '//div[@id="news_content"]')
            item_loader.add_xpath("tags", '//*[@class="news_tags"]/a/text()')
            item_loader.add_xpath("create_data", '//div[@id="news_info"]/*[@class="time"]/text()')
            item_loader.add_value("url", response.url)
            if response.meta.get("front_image_url", ""):
                item_loader.add_value("front_image_url", response.meta.get("front_image_url", ""))

            article_item = item_loader.load_item()

            yield Request(url=parse.urljoin(response.url, "/NewsAjax/GetAjaxNewsInfo?contentId={}".format(post_id)),
                          meta={"article_item":article_item} ,callback=self.parse_nums)

            # praise_nums = j_data["DiggCount"]
            # fav_nums = j_data["TotalView"]
            # comment_nums = j_data["CommentCount"]
        pass

    def parse_nums(self, response):
        j_data = json.loads(response.text)
        article_item = response.meta.get("article_item", "")
        praise_nums = j_data["DiggCount"]
        fav_nums = j_data["TotalView"]
        comment_nums = j_data["CommentCount"]

        article_item["praise_nums"] = praise_nums
        article_item["fav_nums"] = fav_nums
        article_item["comment_nums"] = comment_nums
        article_item["url_object_id"] = common.get_md5(article_item["url"])

        yield article_item
        pass

